import os
import gzip
import re
import time
import sys
import util
import logging
import model
import settings 

logging.basicConfig(filename='log', level=logging.INFO)
Log = logging.getLogger('Main')

def main():
    global Log
    db = model.Database()
    #db = None
    main_clk = time.time()
    totalpage = 0
    totalurl = 0

    Log.info('.....................Start..........................\n')
    Log.info(settings.dirInput)

    # List file in DataTest (hostname == filename)
    for hostid , hostname in zip(range(1,settings.maximumHost),os.listdir(settings.dirInput)):
        start_t = time.time()
        # extract file from .gz 
        try:
            contents = gzip.open(settings.dirInput + hostname ,'rb').read()
            
            # Extract Sites 
            page_list = util.ExtractSites(contents,db)

            Log.info(str(hostid)+' Extract\t' + str(time.time()-start_t)[0:5] + '\t' + hostname )
            totalurl_t = 0
            for starthtml,endhtml,header in page_list:
                start_tt = time.time()
                totalurl_t,uniqurl = util.ExtractURL(contents[starthtml:endhtml],header['url'],db)
                totalurl += totalurl_t

            totalpage += len(page_list)

            Log.info('\t\t' + str(time.time()-start_t)[0:5] + '\t'+str(len(page_list))+' pages\t'+str(totalurl_t)+' urls')

        except IOError :
            pass

    uniqurl = 0
    if db != None:
        uniqurl = db.totalurl
        db.close()
    #                                       Total site             Total page                Total url       
    Log.info('')
    Log.info('>> Summary')
    Log.info('Time usage     : '+str(time.time()-main_clk) +' s')
    Log.info('Total sites    : '+str(hostid)+' hosts')
    Log.info('Total pages    : '+str(totalpage)+' pages')
    Log.info('Total Uniq url : '+str(uniqurl)+' urls')
    Log.info('Avg pages/site : '+str(totalpage*1.0/hostid))
    Log.info('Avg urls/page  : '+str(totalurl*1.0/totalpage))
    Log.info('Avg urls/site  : '+str(totalurl*1.0/hostid))
    Log.info('......................END...........................\n')


if __name__ == "__main__" :
    main()
    os.system('rm *.pyc')
